package com.sena.crawler.util;

import com.alibaba.fastjson.JSON;
import com.sena.crawler.SenaCrawler;
import com.sena.crawler.conf.SenaCrawlerConf;
import com.sena.crawler.domian.*;
import com.sena.crawler.enums.FileTypeEnums;
import com.sena.crawler.enums.LinkTypeEnums;
import com.sena.crawler.model.PageRequest;
import com.sena.crawler.model.RunConf;
//import com.sena.yun.main.SenaFileUrlCrawlerYunMain;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.parser.Tag;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.*;

/**
 * jsoup tool
 *
 * @author seven 2015-05-14 22:44:43
 */
public class JsoupUtil {
    private static Logger logger = LoggerFactory.getLogger(JsoupUtil.class);

    /**
     * 加载页面
     *
     * @param pageRequest
     * @return Document
     */
    public static Document load(PageRequest pageRequest) {
        if (!UrlUtil.isUrl(pageRequest.getUrl())) {
            logger.info("url错误");
            return null;
        }
        try {
            // 请求设置
            Connection conn = Jsoup.connect(pageRequest.getUrl());
            if (pageRequest.getParamMap() != null && !pageRequest.getParamMap().isEmpty()) {
                conn.data(pageRequest.getParamMap());
            }
            if (pageRequest.getCookieMap() != null && !pageRequest.getCookieMap().isEmpty()) {
                conn.cookies(pageRequest.getCookieMap());
            }
            if (pageRequest.getHeaderMap() != null && !pageRequest.getHeaderMap().isEmpty()) {
                conn.headers(pageRequest.getHeaderMap());
            }
            if (pageRequest.getUserAgent() != null) {
                conn.userAgent(pageRequest.getUserAgent());
            }
            if (pageRequest.getReferrer() != null) {
                conn.referrer(pageRequest.getReferrer());
            }
            conn.timeout(pageRequest.getTimeoutMillis());
            conn.validateTLSCertificates(pageRequest.isValidateTLSCertificates());
            conn.maxBodySize(0);    // 取消默认1M限制

            // 代理
            if (pageRequest.getProxy() != null) {
                conn.proxy(pageRequest.getProxy());
            }

            // 发出请求
            Document html = null;
            if (pageRequest.isIfPost()) {
                html = conn.post();
            } else {
                html = conn.get();
            }
            return html;
        } catch (IOException e) {
            logger.error(e.getMessage(), e);
            return null;
        }
    }


    public static String loadPageSource(PageRequest pageRequest) {
        if (!UrlUtil.isUrl(pageRequest.getUrl())) {
            return null;
        }
        try {
            // 请求设置
            Connection conn = Jsoup.connect(pageRequest.getUrl());
            if (pageRequest.getParamMap() != null && !pageRequest.getParamMap().isEmpty()) {
                conn.data(pageRequest.getParamMap());
            }
            if (pageRequest.getCookieMap() != null && !pageRequest.getCookieMap().isEmpty()) {
                conn.cookies(pageRequest.getCookieMap());
            }
            if (pageRequest.getHeaderMap() != null && !pageRequest.getHeaderMap().isEmpty()) {
                conn.headers(pageRequest.getHeaderMap());
            }
            if (pageRequest.getUserAgent() != null) {
                conn.userAgent(pageRequest.getUserAgent());
            }
            if (pageRequest.getReferrer() != null) {
                conn.referrer(pageRequest.getReferrer());
            }
            conn.timeout(pageRequest.getTimeoutMillis());
            conn.validateTLSCertificates(pageRequest.isValidateTLSCertificates());
            conn.maxBodySize(0);    // 取消默认1M限制

            // 代理
            if (pageRequest.getProxy() != null) {
                conn.proxy(pageRequest.getProxy());
            }

            conn.ignoreContentType(true);
            conn.method(pageRequest.isIfPost() ? Connection.Method.POST : Connection.Method.GET);

            // 发出请求
            Connection.Response resp = conn.execute();
            String pageSource = resp.body();
            return pageSource;
        } catch (IOException e) {
            logger.error(e.getMessage(), e);
            return null;
        }
    }

    /**
     * 抽取元素数据
     *
     * @param fieldElement
     * @param selectType
     * @param selectVal
     * @return String
     */
    public static String parseElement(Element fieldElement, SenaCrawlerConf.SelectType selectType, String selectVal) {
        String fieldElementOrigin = null;
        if (SenaCrawlerConf.SelectType.HTML == selectType) {
            fieldElementOrigin = fieldElement.html();
        } else if (SenaCrawlerConf.SelectType.VAL == selectType) {
            fieldElementOrigin = fieldElement.val();
        } else if (SenaCrawlerConf.SelectType.TEXT == selectType) {
            fieldElementOrigin = fieldElement.text();
        } else if (SenaCrawlerConf.SelectType.ATTR == selectType) {
            fieldElementOrigin = fieldElement.attr(selectVal);
        } else if (SenaCrawlerConf.SelectType.HAS_CLASS == selectType) {
            fieldElementOrigin = String.valueOf(fieldElement.hasClass(selectVal));
        } else {
            fieldElementOrigin = fieldElement.toString();
        }
        return fieldElementOrigin;
    }

    /**
     * 获取页面上所有超链接地址 （<a>标签的href值）
     *
     * @param html 页面文档
     * @return Set<String>
     */
    public static Set<Element> findLinks(Document html, RunConf runConf, SenaCrawler crawler) {
        if (html == null) {
            return null;
        }

        Elements hrefElements = html.select("a[href]");

        Set<Element> elementSets = new HashSet<Element>();
        Set<Element> fileItems = new HashSet<>();
        Set<Element> otherItems = new HashSet<>();
        Set<Element> hiddenItems = new HashSet<>();
        // 抽取数据

        if (hrefElements != null && hrefElements.size() > 0) {
            for (Element item : hrefElements) {
                // href、abs:href
                String href = item.attr("abs:href");
                FileTypeEnums fileTypeEnums = null;
                if (UrlUtil.isUrl(href) && !"最末页".equals(item.text())) {
                    try {
                        String code = href.substring(href.lastIndexOf("."));
                        fileTypeEnums = FileTypeEnums.getByCodeType(code);
                    } catch (Exception e) {
                        e.printStackTrace();
                    }
                    //其他站链接
                    if (runConf.validWhiteUrl(href) && !runConf.validBlackUrl(href)) {
                        //不获取pdf等文件内容
                        if (!match(item)) {

                            elementSets.add(item);
                        } else {
                            fileItems.add(item);
                        }
                        if (StringUtils.isEmpty(item.text()) || item.childNodeSize() == 0) {
                            hiddenItems.add(item);
                        }
                    } else {
                        if (StringUtils.isEmpty(item.text()) || item.childNodeSize() == 0) {
                            hiddenItems.add(item);
                        }
                        if (match(item)) {
                            fileItems.add(item);
                        }
                        otherItems.add(item);
                    }
                }
            }
        }
        if (runConf.getSenaCrawlerType().equals(LinkTypeEnums.OUTER_LINK.getCode())) {
            saveOuterToRedis(runConf, otherItems, html.baseUri());
        }
        if (runConf.getSenaCrawlerType().equals(LinkTypeEnums.FILE_LINK.getCode())) {
            saveFileToRedis(runConf, fileItems, html.baseUri(), runConf);
        }
        if (runConf.getSenaCrawlerType().equals(LinkTypeEnums.HIDDEN_LINK.getCode())) {

            saveHiddenToRedis(runConf, hiddenItems, html.baseUri());
        }
        addPurl(html, runConf, crawler);
        return elementSets;
    }


    private static void addPurl(Document html, RunConf runConf, SenaCrawler crawler) {

        Set<String> fileItems = new HashSet<>();
        Set<String> otherItems = new HashSet<>();
        List<String> plist = new ArrayList<>();
        Elements pe = html.select("body");
        for (Element item : pe) {
            List<String> list = RegexUtil.findUrl(item.text());
            if (list != null && list.size() > 0) {
                plist.addAll(list);
            }
        }
        for (String href : plist) {
            //其他站链接
            if (runConf.validWhiteUrl(href) && !runConf.validBlackUrl(href)) {
                //本站的
                //不获取pdf等文件内容
                if (match(href)) {
                    fileItems.add(href);
                } else {
                    Link link = new Link(href, LinkTypeEnums.COMMON_LINK.getCode(), html.baseUri(), "标签页面连接");
                    //处理后缀
                    dealSuffix(link, crawler);
                    //判断是否需要最大数量的url
                    if (crawler.getRunConf().isNeedMax()) {
                        //判断连接是否在需要访问的url中
                        if (crawler.getRunData().isFind(link)) {
                            boolean flag = dealCount(link, crawler);
                            //判断是大于最大的连接数了
                            if (!flag) {

                                if (crawler.getRunData().addUrl(link)) {
                                    //添加到redis
                                    crawler.getRunConf().getJedisUtil().addSetValue(LinkTypeEnums.UN_VISITED_URL.getCode(),link.getLink());
                                    if (LinkTypeEnums.COMMON_LINK.getCode().equals(crawler.getRunConf().getSenaCrawlerType())) {
                                        crawler.getRunConf().getJedisUtil().setSetValue(LinkTypeEnums.COMMON_LINK.getCode(), JSON.toJSONString(link));
                                    }
                                }
                            }
                        }
                    } else {
                        if (crawler.getRunData().addUrl(link)) {
                            //添加到redis
                            crawler.getRunConf().getJedisUtil().addSetValue(LinkTypeEnums.UN_VISITED_URL.getCode(),link.getLink());
                            if (LinkTypeEnums.COMMON_LINK.getCode().equals(crawler.getRunConf().getSenaCrawlerType())) {
                                crawler.getRunConf().getJedisUtil().setSetValue(LinkTypeEnums.COMMON_LINK.getCode(), JSON.toJSONString(link));
                            }
                        }
                    }
                }
            } else {
                //外站点的
                if (match(href)) {
                    fileItems.add(href);
                } else {
                    otherItems.add(href);
                }
                //

            }
        }
        if (runConf.getSenaCrawlerType().equals(LinkTypeEnums.OUTER_LINK.getCode())) {
            saveOuterStringToRedis(runConf, otherItems, html.baseUri());
        }
        if (runConf.getSenaCrawlerType().equals(LinkTypeEnums.FILE_LINK.getCode())) {
            saveFileStrURLToRedis(runConf, fileItems, html.baseUri(), runConf);
        }
    }

    /**
     * 处理数据
     *
     * @param
     * @param
     */
    private static void saveFileToRedis(RunConf conf, Set<Element> item, String plink, RunConf runConf) {
        item.forEach(e -> {
            String href = e.attr("abs:href");
            FileLink fileLink;
            if (conf.getJedisUtil().hExists(LinkTypeEnums.FILE_LINK.getCode(), plink)) {
                fileLink = (FileLink) conf.getJedisUtil().getMapValue(LinkTypeEnums.FILE_LINK.getCode(), plink, FileLink.class);
                if (fileLink.getLinks() != null && !fileLink.getLinks().contains(href)) {
                    Set<String> set = fileLink.getLinks();
                    set.add(href);
                    fileLink.setLink_cnt(set.size());
                    conf.getJedisUtil().setMapValue(LinkTypeEnums.FILE_LINK.getCode(), plink, JSON.toJSONString(fileLink));
                    //SenaFileUrlCrawlerYunMain.saveFileUrlToMysql(runConf.getJdbcUrl(), "root", runConf.getJdbcUserPassWord(), plink, href);
                }
            } else {
                fileLink = new FileLink();
                Set<String> set = new HashSet<>();
                set.add(href);
                fileLink.setLink_cnt(set.size());
                fileLink.setScan_time(new Date());
                fileLink.setScan_url(plink);
                fileLink.setLinks(set);
                conf.getJedisUtil().setMapValue(LinkTypeEnums.FILE_LINK.getCode(), plink, JSON.toJSONString(fileLink));
                //SenaFileUrlCrawlerYunMain.saveFileUrlToMysql(runConf.getJdbcUrl(), "root", runConf.getJdbcUserPassWord(), plink, href);
            }
        });
    }

    private static void saveFileStrURLToRedis(RunConf conf, Set<String> item, String plink, RunConf runConf) {
        item.forEach(e -> {
            String href = e;
            FileLink fileLink;
            if (conf.getJedisUtil().hExists(LinkTypeEnums.FILE_LINK.getCode(), plink)) {
                fileLink = (FileLink) conf.getJedisUtil().getMapValue(LinkTypeEnums.FILE_LINK.getCode(), plink, FileLink.class);
                if (fileLink.getLinks() != null && !fileLink.getLinks().contains(href)) {
                    Set<String> set = fileLink.getLinks();
                    set.add(href);
                    fileLink.setLink_cnt(set.size());
                    conf.getJedisUtil().setMapValue(LinkTypeEnums.FILE_LINK.getCode(), plink, JSON.toJSONString(fileLink));
                    //SenaFileUrlCrawlerYunMain.saveFileUrlToMysql(runConf.getJdbcUrl(), "root", runConf.getJdbcUserPassWord(), plink, href);
                }
            } else {
                fileLink = new FileLink();
                Set<String> set = new HashSet<>();
                set.add(href);
                fileLink.setLink_cnt(set.size());
                fileLink.setScan_time(new Date());
                fileLink.setScan_url(plink);
                fileLink.setLinks(set);
                conf.getJedisUtil().setMapValue(LinkTypeEnums.FILE_LINK.getCode(), plink, JSON.toJSONString(fileLink));
                //SenaFileUrlCrawlerYunMain.saveFileUrlToMysql(runConf.getJdbcUrl(), "root", runConf.getJdbcUserPassWord(), plink, href);
            }
        });
    }

    /**
     * 保持数据到redis
     *
     * @param item
     */
    private static void saveHiddenToRedis(RunConf conf, Set<Element> item, String plink) {
        item.forEach(e -> {
            String href = e.attr("abs:href");
            if (StringUtils.isNotBlank(href) && UrlUtil.isUrl(href)) {
                //判断是隐藏连接

                HiddenLink link = null;
                if (conf.getJedisUtil().hExists(LinkTypeEnums.HIDDEN_LINK.getCode(), plink)) {
                    link = (HiddenLink) conf.getJedisUtil().getMapValue(LinkTypeEnums.HIDDEN_LINK.getCode(), plink, HiddenLink.class);
                    Set<String> set = link.getAlinks();
                    if (!set.contains(href)) {
                        set.add(href);
                        link.setCnt_alink(set.size());
                        link.setScan_time(new Date());
                        conf.getJedisUtil().setMapValue(LinkTypeEnums.HIDDEN_LINK.getCode(), plink, JSON.toJSONString(link));
                    }
                } else {

                    Set<String> set = new HashSet<>();
                    set.add(href);
                    link = new HiddenLink(new Date(), plink, set.size(), set);
                    conf.getJedisUtil().setMapValue(LinkTypeEnums.HIDDEN_LINK.getCode(), plink, JSON.toJSONString(link));
                }

            }
        });
    }

    /**
     * 提取外站数据
     *
     * @param items
     */
    private static void saveOuterToRedis(RunConf conf, Set<Element> items, String plink) {
        items.forEach(e -> {
            String href = e.attr("abs:href");
            if (StringUtils.isNotBlank(href) && UrlUtil.isUrl(href)) {
                //处理是连接
                OuterLink link = null;
                if (conf.getJedisUtil().hExists(LinkTypeEnums.OUTER_LINK.getCode(), plink)) {
                    link = (OuterLink) conf.getJedisUtil().getMapValue(LinkTypeEnums.OUTER_LINK.getCode(), plink, OuterLink.class);
                    Set<String> set = link.getWlinks();
                    if (!set.contains(href)) {
                        set.add(href);
                        link.setCnt_wlink(set.size());
                        link.setScan_time(new Date());
                        conf.getJedisUtil().setMapValue(LinkTypeEnums.OUTER_LINK.getCode(), plink, JSON.toJSONString(link));
                    }
                } else {
                    Set<String> set = new HashSet<>();
                    set.add(href);
                    link = new OuterLink(new Date(), plink, set.size(), set);
                    conf.getJedisUtil().setMapValue(LinkTypeEnums.OUTER_LINK.getCode(), plink, JSON.toJSONString(link));
                }
            }
        });

    }


    private static void saveOuterStringToRedis(RunConf conf, Set<String> items, String plink) {
        items.forEach(e -> {
            String href = e;
            if (StringUtils.isNotBlank(href) && UrlUtil.isUrl(href)) {
                //处理是连接
                OuterLink link = null;
                if (conf.getJedisUtil().hExists(LinkTypeEnums.OUTER_LINK.getCode(), plink)) {
                    link = (OuterLink) conf.getJedisUtil().getMapValue(LinkTypeEnums.OUTER_LINK.getCode(), plink, OuterLink.class);
                    Set<String> set = link.getWlinks();
                    if (!set.contains(href)) {
                        set.add(href);
                        link.setCnt_wlink(set.size());
                        link.setScan_time(new Date());
                        conf.getJedisUtil().setMapValue(LinkTypeEnums.OUTER_LINK.getCode(), plink, JSON.toJSONString(link));
                    }
                } else {
                    Set<String> set = new HashSet<>();
                    set.add(href);
                    link = new OuterLink(new Date(), plink, set.size(), set);
                    conf.getJedisUtil().setMapValue(LinkTypeEnums.OUTER_LINK.getCode(), plink, JSON.toJSONString(link));
                }
            }
        });

    }


    /**
     * 获取页面上所有图片地址 （<a>标签的href值）
     *
     * @param html
     * @return Set<String>
     */
    public static Set<String> findImages(Document html) {

        Elements imgs = html.getElementsByTag("img");

        Set<String> images = new HashSet<String>();
        if (imgs != null && imgs.size() > 0) {
            for (Element element : imgs) {
                String imgSrc = element.attr("abs:src");
                images.add(imgSrc);
            }
        }
        return images;
    }

    public static boolean match(Element element) {
        String href = element.attr("abs:href");
        String code = href.substring(href.lastIndexOf("."));
        FileTypeEnums fileTypeEnums = FileTypeEnums.getByCodeType(code);
        if (fileTypeEnums != null) {
            return true;
        }
        String endStr = element.text();
        if (endStr.contains(".")) {
            String type = endStr.substring(endStr.lastIndexOf("."));
            if (FileTypeEnums.getByCodeType(type) != null) {
                return true;
            }
        }
        return false;
    }

    public static boolean match(String href) {
        String code = href.substring(href.lastIndexOf("."));
        FileTypeEnums fileTypeEnums = FileTypeEnums.getByCodeType(code);
        if (fileTypeEnums != null) {
            return true;
        }
        return false;
    }

    private  static void dealSuffix(Link link, SenaCrawler crawler) {
        if (crawler.getRunConf().getSuffixs().size() > 0) {
            HashSet<String> strings = (HashSet<String>) crawler.getRunConf().getSuffixs();
            strings.forEach(e -> {
                if (link.getLink().contains(e)) {
                    if (link.getLink().contains("?" + e) || link.getLink().contains("&" + e)) {
                        String str = link.getLink().substring(0, link.getLink().lastIndexOf(e) - 1);
                        link.setLink(str);
                    } else {
                        String str = link.getLink().substring(0, link.getLink().lastIndexOf(e));
                        link.setLink(str);
                    }
                }
            });
        }
    }

    private static boolean dealCount(Link link, SenaCrawler crawler) {
        if (!link.getLink().equals(crawler.getRunConf().getRootLink())) {
            String url = link.getLink();
            String rel = null;
            if (url.contains("?")) {
                rel = url.substring(0, url.lastIndexOf("?"));
            } else {
                rel = url;
            }
            LinkCount linkCount = null;
            if (crawler.getRunConf().getJedisUtil().hExists(LinkTypeEnums.COUNT_LINK.getCode(), rel)) {
                linkCount = (LinkCount) crawler.getRunConf().getJedisUtil().getMapValue(LinkTypeEnums.COUNT_LINK.getCode(), rel, LinkCount.class);
                if (linkCount.getCount() >= crawler.getRunConf().getMaxUrl()) {
                    return true;
                } else {
                    linkCount.setCount(linkCount.getCount() + 1);
                }
            } else {
                linkCount = new LinkCount();
                linkCount.setUrl(rel);
                linkCount.setCount(1);
            }
            crawler.getRunConf().getJedisUtil().setMapValue(LinkTypeEnums.COUNT_LINK.getCode(), rel, JSON.toJSONString(linkCount));
            return false;
        } else {
            return false;
        }
    }


}
